From: Alex Crichton Date: Fri, 2 Jun 2017 13:57:43 +0000 (-0700) Subject: Optimize allocations in the registry index X-Git-Tag: archive/raspbian/0.35.0-2+rpi1~3^2^2^2^2^2^2^2~22^2~9^2~8^2~13 X-Git-Url: https://dgit.raspbian.org/%22http://www.example.com/cgi/success//%22http:/www.example.com/cgi/success/?a=commitdiff_plain;h=d303306d375241d2eb424c8a00663497e539fbc3;p=cargo.git Optimize allocations in the registry index Avoid some unnecessary clones, `to_vec`, etc. Not super optimizal but improves the profile here regardless. --- diff --git a/src/cargo/sources/registry/index.rs b/src/cargo/sources/registry/index.rs index b92f717b5..413502467 100644 --- a/src/cargo/sources/registry/index.rs +++ b/src/cargo/sources/registry/index.rs @@ -63,9 +63,6 @@ impl<'cfg> RegistryIndex<'cfg> { return Ok(&self.cache[name]); } let summaries = self.load_summaries(name, load)?; - let summaries = summaries.into_iter().filter(|summary| { - summary.0.package_id().name() == name - }).collect(); self.cache.insert(name.to_string(), summaries); Ok(&self.cache[name]) } @@ -98,31 +95,38 @@ impl<'cfg> RegistryIndex<'cfg> { 2 => format!("2/{}", fs_name), 3 => format!("3/{}/{}", &fs_name[..1], fs_name), _ => format!("{}/{}/{}", &fs_name[0..2], &fs_name[2..4], fs_name), - // 1 => Path::new("1").join(fs_name), - // 2 => Path::new("2").join(fs_name), - // 3 => Path::new("3").join(&fs_name[..1]).join(fs_name), - // _ => Path::new(&fs_name[0..2]).join(&fs_name[2..4]).join(fs_name), }; - match load.load(&root, Path::new(&path)) { - Ok(contents) => { - let contents = str::from_utf8(&contents).map_err(|_| { - CargoError::from("registry index file was not valid utf-8") - })?; - let lines = contents.lines() - .map(|s| s.trim()) - .filter(|l| !l.is_empty()); - - // Attempt forwards-compatibility on the index by ignoring - // everything that we ourselves don't understand, that should - // allow future cargo implementations to break the - // interpretation of each line here and older cargo will simply - // ignore the new lines. - Ok(lines.filter_map(|line| { - self.parse_registry_package(line).ok() - }).collect()) - } - Err(..) => Ok(Vec::new()), + let mut ret = Vec::new(); + let mut hit_closure = false; + let err = load.load(&root, Path::new(&path), &mut |contents| { + hit_closure = true; + let contents = str::from_utf8(contents).map_err(|_| { + CargoError::from("registry index file was not valid utf-8") + })?; + let lines = contents.lines() + .map(|s| s.trim()) + .filter(|l| !l.is_empty()); + + // Attempt forwards-compatibility on the index by ignoring + // everything that we ourselves don't understand, that should + // allow future cargo implementations to break the + // interpretation of each line here and older cargo will simply + // ignore the new lines. + ret.extend(lines.filter_map(|line| { + self.parse_registry_package(line).ok() + })); + + Ok(()) + }); + + // We ignore lookup failures as those are just crates which don't exist + // or we haven't updated the registry yet. If we actually ran the + // closure though then we care about those errors. + if hit_closure { + err?; } + + Ok(ret) } /// Parse a line from the registry's index file into a Summary for a diff --git a/src/cargo/sources/registry/local.rs b/src/cargo/sources/registry/local.rs index ef23552ca..203f3bc25 100644 --- a/src/cargo/sources/registry/local.rs +++ b/src/cargo/sources/registry/local.rs @@ -36,8 +36,11 @@ impl<'cfg> RegistryData for LocalRegistry<'cfg> { &self.index_path } - fn load(&self, root: &Path, path: &Path) -> CargoResult> { - paths::read_bytes(&root.join(path)) + fn load(&self, + root: &Path, + path: &Path, + data: &mut FnMut(&[u8]) -> CargoResult<()>) -> CargoResult<()> { + data(&paths::read_bytes(&root.join(path))?) } fn config(&mut self) -> CargoResult> { diff --git a/src/cargo/sources/registry/mod.rs b/src/cargo/sources/registry/mod.rs index 66f267a79..24b5daf96 100644 --- a/src/cargo/sources/registry/mod.rs +++ b/src/cargo/sources/registry/mod.rs @@ -221,7 +221,10 @@ struct RegistryDependency<'a> { pub trait RegistryData { fn index_path(&self) -> &Filesystem; - fn load(&self, root: &Path, path: &Path) -> CargoResult>; + fn load(&self, + _root: &Path, + path: &Path, + data: &mut FnMut(&[u8]) -> CargoResult<()>) -> CargoResult<()>; fn config(&mut self) -> CargoResult>; fn update_index(&mut self) -> CargoResult<()>; fn download(&mut self, diff --git a/src/cargo/sources/registry/remote.rs b/src/cargo/sources/registry/remote.rs index cddbda2ed..37b776b6d 100644 --- a/src/cargo/sources/registry/remote.rs +++ b/src/cargo/sources/registry/remote.rs @@ -130,7 +130,10 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> { &self.index_path } - fn load(&self, _root: &Path, path: &Path) -> CargoResult> { + fn load(&self, + _root: &Path, + path: &Path, + data: &mut FnMut(&[u8]) -> CargoResult<()>) -> CargoResult<()> { // Note that the index calls this method and the filesystem is locked // in the index, so we don't need to worry about an `update_index` // happening in a different process. @@ -142,7 +145,7 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> { Some(blob) => blob, None => bail!("path `{}` is not a blob in the git repo", path.display()), }; - Ok(blob.content().to_vec()) + data(blob.content()) } fn config(&mut self) -> CargoResult> { @@ -150,9 +153,12 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> { let _lock = self.index_path.open_ro(Path::new(INDEX_LOCK), self.config, "the registry index")?; - let json = self.load(Path::new(""), Path::new("config.json"))?; - let config = serde_json::from_slice(&json)?; - Ok(Some(config)) + let mut config = None; + self.load(Path::new(""), Path::new("config.json"), &mut |json| { + config = Some(serde_json::from_slice(&json)?); + Ok(()) + })?; + Ok(config) } fn update_index(&mut self) -> CargoResult<()> {